﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Collections;
using System.IO;


namespace ConsoleApplication1
{
    class Parser
    {
        //private enum QuoteStatus { FIRST, LAST , MIDDLE , NO };
        private static Dictionary<string, string> m_AllDocs; // the large term dictionary that we will use in Form1 for outputing results
        private static Hashtable stopWordsHash; // hastable of stopwords read from stopwords.txt
        private static bool useStemmer;
        public static Stemmer stemmer;

        private static char[] delimiters = { ' ','\r', '\n', '\t', '.', '\"', '/', ';', ',', '?', '!', ')', '(', '[', ']', '{',
                                         '}','�', '-', '—', '_', '+', '=', '*', '&', '\'', '`', ':', '~', '%', '^', '|', '׀', '@', '#', '$', '?', '<', '>' };
        private static string[] leftOvers = new string[] { "", ",", "-", " " };
        private static string[] months = new string[] { "january", "february", "march", "april", "may", "june", "july", "august", "september", " october", "november", "december" };
        
        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="parse">bool variable that determines whether to do stemming or not</param>
        public Parser(bool parse)
        {
            stemmer = new Stemmer();
            useStemmer = parse;
            m_AllDocs = new Dictionary<string, string>();
            stopWordsHash = new Hashtable();
            Stopwords(); // Read the stopwords from stopwords.text
        }

        
        /// <summary>
        /// Reads all the stopwords from a file called stopwords.txt
        /// </summary>
        private void Stopwords()
        {
            StreamReader inputFile = File.OpenText("stopWords.txt");
            string stopword = inputFile.ReadLine();
            stopword = stopword.ToLower();
            while (!inputFile.EndOfStream)
            {
                if (!stopWordsHash.Contains(stopword))
                {
                    stopWordsHash.Add(stopword, stopword); // Using hastable for quick search
                }
                stopword = inputFile.ReadLine();
                stopword = stopword.ToLower();
            }
            inputFile.Close();
        }


        public static Dictionary<string, string> ProcessFiles(Dictionary<string, string> allDocuments)
        {
            foreach (string currentName in allDocuments.Keys)
            {
                string allTermsOfFile = ""; //keeps all the terms (words) of the file
                string text = allDocuments[currentName].ToLower();
                string[] split = text.Split(new char[] { ' ', '\n', '\r', '\t', '(', '[', '-' }, StringSplitOptions.RemoveEmptyEntries); // Split the text based on those delimiters

                for (int i = 0; i < split.Length; i++) // run on the entire array of words (after split)
                {
                    string current = split[i];
                    int num;
                    string temp = "";
                    double num1;
                    if (double.TryParse(current, out num1)) // try converting the word into a number (num1 contains the number if it is a number) 
                        current = num1.ToString(); // if succesful , convert back to a number (this will keep a constant format between all types of numbers , for example : 1,200 will become 1200)

                    if (months.Contains(current)) // check if the word is one of the 12 months
                    {
                        if (i + 1 < split.Length) // check that we are not in the end of the array of words
                        {
                            temp = split[i + 1].Trim(delimiters);  // remove trailing signs (example . , ! and so on)
                            if (int.TryParse(temp, out num) && (num >= 1 && num <= 31)) // try converting the next word into a number , if succesful and the number is between 1 and 31 chain it to the current word
                            {
                                current += " " + split[i + 1]; // the chain will include the trailing signs (for example "31," and not just "31")
                                i++;
                            }
                        }
                        if (i + 1 < split.Length) // check that we are not in the end of the array of words
                        {
                            temp = split[i + 1].Trim(delimiters);
                            if (int.TryParse(temp, out num) && (num > 0 && num < 5000)) // try converting the next word into a number , if succesful and the number is between 0 and 5000 (random number expressing the largest year expected) chain it to the current word
                            {
                                current += " " + split[i + 1];
                                i++;
                            }
                        }
                    }

                    current = current.Trim(delimiters); // remove all trailing signs
                    if (leftOvers.Contains(current) || stopWordsHash.Contains(current)) // check if the word is an empty sign or a stop word (Trim removes signs (like ! , ?) from the start or the end of the word)
                        continue; // if yes skip to the next index in the array
                    if (useStemmer) // check if there was a request to use the stemmer or not
                    {
                        stemmer.add(current.ToCharArray(), current.Length); // use the stemmer
                        stemmer.stem();
                        current = stemmer.ToString();
                        current = current.Trim(delimiters); // remove all trailing signs left after stemmer
                    }

                    if (i + 1 < split.Length)
                        allTermsOfFile += current + " "; //concatenate current term to all terms
                    else
                        allTermsOfFile += current;
                }//for 
                m_AllDocs.Add(currentName, allTermsOfFile); // if not , add it to the dictionary with first appearance
                //   Console.WriteLine(fileName + "\n" + allTermsOfFile + "\n");
            }
            return m_AllDocs;
        }//ProcessFiles


        /*
         * BACKUP before Boris's copy
         *          
              /// <summary>
              /// for each file: delimiters  and stopWords will be deleted. 
              /// Stemming will be done according to the input in "config.txt"
              /// </summary>
              /// <param name="allDocs">all the documents, where KEY=fileName and VALUE=stringFile</param>
              public static Dictionary<string, string> ProcessFiles(Dictionary<string, string> allDocs)
           {
                  foreach (string fileName in allDocs.Keys)
                  {
                      string allTermsOfFile = ""; //keeps all the terms (words) of the file
                      string text = allDocs[fileName].ToLower();
                      string[] split = text.Split(new char[] { ' ', '\n', '\r', '\t', '(', '[', '-' }, StringSplitOptions.RemoveEmptyEntries); // Split the text based on those delimiters
                      for (int i = 0; i < split.Length; i++)
                      {
                          string current = split[i];
                          current = current.Trim(delimiters); // remove all trailing signs
                          if (current == "it’s")
                              Console.WriteLine("it's");
                          if (leftOvers.Contains(current) || stopWordsHash.Contains(current)) // check if the word is an empty sign or a stop word (Trim removes signs (like ! , ?) from the start or the end of the word)
                              continue; // if yes skip to the next index in the array

                          if (useStemmer) // check if there was a request to use the stemmer or not
                          {
                              stemmer.add(current.ToCharArray(), current.Length); // use the stemmer
                              stemmer.stem();
                              current = stemmer.ToString();
                              current = current.Trim(delimiters); // remove all trailing signs left after stemmer
                          }
                          if (i + 1 < split.Length)
                              allTermsOfFile += current + " "; //concatenate current term to all terms
                          else
                              allTermsOfFile += current;
                      }
                      m_AllDocs.Add(fileName, allTermsOfFile); // if not , add it to the dictionary with first appearance
                   //   Console.WriteLine(fileName + "\n" + allTermsOfFile + "\n");
                
                  }
                  return m_AllDocs;
             }
         *
         * 
         */


        /// <summary>
        /// Parses a document (string given as paramater) and return a dictionary with terms and their number of appearances
        /// </summary>
        /// <param name="docu"></param>
        /// <returns></returns>
        //public static Dictionary<string, string> ProcessFiles(Dictionary<string, string> allDocuments)
        //{

        //    foreach (string currentName in allDocuments.Keys)
        //    {
        //        string allTermsOfFile = ""; //keeps all the terms (words) of the file
        //        int start = allDocuments[currentName].IndexOf("<TEXT>"); // find the index in the string of the word <TEXT>
        //        int end = allDocuments[currentName].IndexOf("</TEXT>"); // find the index in the string of the word </TEXT>
        //        string text = allDocuments[currentName].Substring(start + 6, end - start - 6); // get the text starting from the index of <TEXT> and length is the difference between start and end.
        //        start = allDocuments[currentName].IndexOf("<HEADLINE>");
        //        end = allDocuments[currentName].IndexOf("</HEADLINE>");
        //        string headline = allDocuments[currentName].Substring(start + 10, end - start - 10);
        //        text = text.ToLower(); // convert all the text to Lowercase
        //        int count = 1;
        //        string[] split = text.Split(new char[] { ' ', '\n', '\r', '\t', '(', '[', '-' }, StringSplitOptions.RemoveEmptyEntries); // Split the text based on those delimiters
        //        for (int i = 0; i < split.Length; i++) // run on the entire array of words (after split)
        //        {
        //            string current = split[i];
        //            int num;
        //            string temp = "";
        //            double num1;
        //            if (double.TryParse(current, out num1)) // try converting the word into a number (num1 contains the number if it is a number) 
        //                current = num1.ToString(); // if succesful , convert back to a number (this will keep a constant format between all types of numbers , for example : 1,200 will become 1200)

        //            if (months.Contains(current)) // check if the word is one of the 12 months
        //            {
        //                if (i + 1 < split.Length) // check that we are not in the end of the array of words
        //                {
        //                    temp = split[i + 1].Trim(delimiters);  // remove trailing signs (example . , ! and so on)
        //                    if (int.TryParse(temp, out num) && (num >= 1 && num <= 31)) // try converting the next word into a number , if succesful and the number is between 1 and 31 chain it to the current word
        //                    {
        //                        current += " " + split[i + 1]; // the chain will include the trailing signs (for example "31," and not just "31")
        //                        i++;
        //                    }
        //                }
        //                if (i + 1 < split.Length) // check that we are not in the end of the array of words
        //                {
        //                    temp = split[i + 1].Trim(delimiters);
        //                    if (int.TryParse(temp, out num) && (num > 0 && num < 5000)) // try converting the next word into a number , if succesful and the number is between 0 and 5000 (random number expressing the largest year expected) chain it to the current word
        //                    {
        //                        current += " " + split[i + 1];
        //                        i++;
        //                    }
        //                }
        //            }

        //            current = current.Trim(delimiters); // remove all trailing signs
        //            if (leftOvers.Contains(current) || stopWordsHash.Contains(current)) // check if the word is an empty sign or a stop word (Trim removes signs (like ! , ?) from the start or the end of the word)
        //                continue; // if yes skip to the next index in the array
        //            if (useStemmer) // check if there was a request to use the stemmer or not
        //            {
        //                stemmer.add(current.ToCharArray(), current.Length); // use the stemmer
        //                stemmer.stem();
        //                current = stemmer.ToString();
        //                current = current.Trim(delimiters); // remove all trailing signs left after stemmer
        //            }

        //             if (i + 1 < split.Length)
        //                 allTermsOfFile += current + " "; //concatenate current term to all terms
        //             else
        //                 allTermsOfFile += current;
        //              }
        //              m_AllDocs.Add(currentName, allTermsOfFile); // if not , add it to the dictionary with first appearance
        //           //   Console.WriteLine(fileName + "\n" + allTermsOfFile + "\n");
                
        //          }
        //          return m_AllDocs;


        //    }





    }
}
